一.教育数据一共是ibrd的两个库,一个是sourcecode2一个是sourcecode12,先处理新的需求
1.ManyLineToOneLine
inpath
file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/EducationStatistics.json
outpath
/Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/EducationStatistics

inpath
"file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/WorldDevelopmentIndicators.json"
outpath
"/Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/WorldDevelopmentIndicators"

2.detail表有个新的，需要导到hbase里
  [1]先把这个数据跑1.转成一行的json
      val inpath = "file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/Detaildic_IBRD.json"
      val outpath = "file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/Detaildic"
  [2]写入hbase
   Map04_Detail
   inpath="file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/Detaildic/part-00000"
3.transferValueCode_to_Name_IBRD
    val inpath = "file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/EducationStatistics/part-00000"
    val outpath = "file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/EducationStatistics_1"
    val inpath = "file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/WorldDevelopmentIndicators/part-00000"
    val outpath = "file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/WorldDevelopmentIndicators_1"
4.ToJson 转换成json串
    val inpath = "file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/EducationStatistics_1/part-00000"
    val outpath_json = "file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/EducationStatistics_ToJson"
    val inpath = "file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/WorldDevelopmentIndicators_1/part-00000"
    val outpath_json = "file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/WorldDevelopmentIndicators_ToJson"

5.JiaoYu_CSV--教育的结果
  val input_original = "file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/EducationStatistics_ToJson/part-00000"
  al outpath="/Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/EducationStatistics_JIAOYU"
  val input_original = "file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/WorldDevelopmentIndicators_ToJson/part-00000"
  val outpath="/Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/WorldDevelopmentIndicators_JIAOYU"

还需要跑三个老版本的数据
合并文件
cat /Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/EducationStatistics_ToJson/part-00000 > /Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/trans
cat /Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/WorldDevelopmentIndicators_ToJson/part-00000 >> /Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/trans

1.CSYD_TABLEMETA_WORLD
    spark.sql("select * from table").createTempView("table1")
    val inputpath_json="file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/trans"
    val inputpath_file="file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/address3.txt"
    val outpath_sql="file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/CSYD_TABLEMETA_WORLD/"
        spark.sql("select * from table").createTempView("table1")
2.CSYD_TABLEMETA_WORLDJsonToREC
    val inpath="file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/CSYD_TABLEMETA_WORLD/part-00000-350c739a-45f1-4971-a2e6-fe2683e28cb5-c000.json"
    val outpath_result = "file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/CSYD_TABLEMETA_WORLDJsonToREC_RESULT"
    val json_file: RDD[String]=sc.textFile(inpath)
3.YEARBOOKTABLEDATA2008_FORMAL_GUOJI
    val inputpath_json="file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/trans"
    val inputpath_file="file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/address3.txt"
    val outpath_sql="file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/YEARBOOKTABLEDATA2008_FORMAL_GUOJI/"
    spark.sql("select * from table").createTempView("table1")
    val dataframe=spark.read.json(inputpath_json)
    在新的"spark_sql"里删掉了原来的一些字段
4.YEARBOOKTABLEDATA2008_FORMAL_GUOJIJsonToREC
    val inpath="file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/YEARBOOKTABLEDATA2008_FORMAL_GUOJI/part-00000-353eaa5d-b575-4402-a469-877f299afcdf-c000.json"
    val outpath_result = "file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/CSYD_TABLEMETA_WORLDJsonToREC_RESULT"


5.YEARBOOKTABLEDATA2008_FORMAL_REPORTGROUPER_GUOJI_2019_ZL
    spark.sql("select * from table").createTempView("table1")
    val dataframe=spark.read.json(inputpath_json)
    val inputpath_json="file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/trans"
    val inputpath_file="file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/address3.txt"
    val outpath_sql="file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/YEARBOOKTABLEDATA2008_FORMAL_REPORTGROUPER_GUOJI_2019_ZL/"

6.YEARBOOKTABLEDATA2008_FORMAL_REPORTGROUPER_GUOJI_2019_ZLJsonToREC
    val inputpath_sql="file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/YEARBOOKTABLEDATA2008_FORMAL_REPORTGROUPER_GUOJI_2019_ZL/part-00000-e8b3cbc5-f468-4833-b9c1-ab5da2c7f1ef-c000.json"
    val outpath_result="file:///Users/kelisiduofu/IdeaProjects/zhiwang_file/jiaoyu/YEARBOOKTABLEDATA2008_FORMAL_REPORTGROUPER_GUOJI_2019_ZL_RESULT"